from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException
from selenium import webdriver
import time
import pandas as pd


def get_jobs(keyword, num_jobs, verbose, path, slp_time):
    
    '''Gathers jobs as a dataframe, scraped from Glassdoor'''
    
    #Initializing the webdriver
    options = webdriver.ChromeOptions()
    
    #Uncomment the line below if you'd like to scrape without a new Chrome window every time.
    #options.add_argument('headless')
    
    #Change the path to where chromedriver is in your home folder.
    driver = webdriver.Chrome(executable_path=path, options=options)
    driver.set_window_size(1120, 1000)
    
    url = "https://www.glassdoor.com/Job/jobs.htm?suggestCount=0&suggestChosen=false&clickSource=searchBtn&typedKeyword="+keyword+"&sc.keyword="+keyword+"&locT=&locId=&jobType="
    #url = 'https://www.glassdoor.com/Job/jobs.htm?sc.keyword="' + keyword + '"&locT=C&locId=1147401&locKeyword=San%20Francisco,%20CA&jobType=all&fromAge=-1&minSalary=0&includeNoSalaryJobs=true&radius=100&cityId=-1&minRating=0.0&industryId=-1&sgocId=-1&seniorityType=all&companyId=-1&employerSizes=0&applicationType=0&remoteWorkType=0'
    driver.get(url)
    jobs = []

    while len(jobs) < num_jobs:  #If true, should be still looking for new jobs.

        #Let the page load. Change this number based on your internet speed.
        #Or, wait until the webpage is loaded, instead of hardcoding it.
        time.sleep(slp_time)

        #Test for the "Sign Up" prompt and get rid of it.
        try:
            driver.find_element_by_class_name("selected").click()
        except ElementClickInterceptedException:
            pass

        time.sleep(.1)

        try:
            driver.find_element_by_css_selector('[alt="Close"]').click() #clicking to the X.
            print(' x out worked')
        except NoSuchElementException:
            print(' x out failed')
            pass

        
        #Going through each job in this page
        job_buttons = driver.find_elements_by_class_name("jl")  #jl for Job Listing. These are the buttons we're going to click.
        for job_button in job_buttons:  

            print("Progress: {}".format("" + str(len(jobs)) + "/" + str(num_jobs)))
            if len(jobs) >= num_jobs:
                break

            job_button.click()  #You might 
            time.sleep(1)
            collected_successfully = False
            
            while not collected_successfully:
                try:
                    company_name = driver.find_element_by_xpath('.//div[@class="employerName"]').text
                    location = driver.find_element_by_xpath('.//div[@class="location"]').text
                    job_title = driver.find_element_by_xpath('.//div[contains(@class, "title")]').text
                    job_description = driver.find_element_by_xpath('.//div[@class="jobDescriptionContent desc"]').text
                    collected_successfully = True
                except:
                    time.sleep(5)

            try:
                salary_estimate = driver.find_element_by_xpath('.//span[@class="gray salary"]').text
            except NoSuchElementException:
                salary_estimate = -1 #You need to set a "not found value. It's important."
            
            try:
                rating = driver.find_element_by_xpath('.//span[@class="rating"]').text
            except NoSuchElementException:
                rating = -1 #You need to set a "not found value. It's important."

            #Printing for debugging
            if verbose:
                print("Job Title: {}".format(job_title))
                print("Salary Estimate: {}".format(salary_estimate))
                print("Job Description: {}".format(job_description[:500]))
                print("Rating: {}".format(rating))
                print("Company Name: {}".format(company_name))
                print("Location: {}".format(location))

            #Going to the Company tab...
            #clicking on this:
            #
Company
            try:
                driver.find_element_by_xpath('.//div[@class="tab" and @data-tab-type="overview"]').click()

                try:
                    #

                    #    Headquarters
                    #    San Francisco, CA
                    #
                    headquarters = driver.find_element_by_xpath('.//div[@class="infoEntity"]//label[text()="Headquarters"]//following-sibling::*').text
                except NoSuchElementException:
                    headquarters = -1

                try:
                    size = driver.find_element_by_xpath('.//div[@class="infoEntity"]//label[text()="Size"]//following-sibling::*').text
                except NoSuchElementException:
                    size = -1

                try:
                    founded = driver.find_element_by_xpath('.//div[@class="infoEntity"]//label[text()="Founded"]//following-sibling::*').text
                except NoSuchElementException:
                    founded = -1

                try:
                    type_of_ownership = driver.find_element_by_xpath('.//div[@class="infoEntity"]//label[text()="Type"]//following-sibling::*').text
                except NoSuchElementException:
                    type_of_ownership = -1

                try:
                    industry = driver.find_element_by_xpath('.//div[@class="infoEntity"]//label[text()="Industry"]//following-sibling::*').text
                except NoSuchElementException:
                    industry = -1

                try:
                    sector = driver.find_element_by_xpath('.//div[@class="infoEntity"]//label[text()="Sector"]//following-sibling::*').text
                except NoSuchElementException:
                    sector = -1

                try:
                    revenue = driver.find_element_by_xpath('.//div[@class="infoEntity"]//label[text()="Revenue"]//following-sibling::*').text
                except NoSuchElementException:
                    revenue = -1

                try:
                    competitors = driver.find_element_by_xpath('.//div[@class="infoEntity"]//label[text()="Competitors"]//following-sibling::*').text
                except NoSuchElementException:
                    competitors = -1

            except NoSuchElementException:  #Rarely, some job postings do not have the "Company" tab.
                headquarters = -1
                size = -1
                founded = -1
                type_of_ownership = -1
                industry = -1
                sector = -1
                revenue = -1
                competitors = -1

                
            if verbose:
                print("Headquarters: {}".format(headquarters))
                print("Size: {}".format(size))
                print("Founded: {}".format(founded))
                print("Type of Ownership: {}".format(type_of_ownership))
                print("Industry: {}".format(industry))
                print("Sector: {}".format(sector))
                print("Revenue: {}".format(revenue))
                print("Competitors: {}".format(competitors))
                print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")

            jobs.append({"Job Title" : job_title,
            "Salary Estimate" : salary_estimate,
            "Job Description" : job_description,
            "Rating" : rating,
            "Company Name" : company_name,
            "Location" : location,
            "Headquarters" : headquarters,
            "Size" : size,
            "Founded" : founded,
            "Type of ownership" : type_of_ownership,
            "Industry" : industry,
            "Sector" : sector,
            "Revenue" : revenue,
            "Competitors" : competitors})
            #add job to jobs
            
            
        #Clicking on the "next page" button
        try:
            driver.find_element_by_xpath('.//li[@class="next"]//a').click()
        except NoSuchElementException:
            print("Scraping terminated before reaching target number of jobs. Needed {}, got {}.".format(num_jobs, len(jobs)))
            break

    return pd.DataFrame(jobs)  #This line converts the dictionary object into a pandas DataFrame.

##Data Collection
import glassdoor_scraper as gs 
import pandas as pd 
path = "C:/Users/Monica/Desktop/salary_prediction_app/chromedriver.exe"

df = gs.get_jobs('data scientist',1000, False, path, 15)

df.to_csv('glassdoor_jobs.csv', index = False)    

import pandas as pd 

df = pd.read_csv('glassdoor_jobs.csv')

#salary parsing 

df['hourly'] = df['Salary Estimate'].apply(lambda x: 1 if 'per hour' in x.lower() else 0)
df['employer_provided'] = df['Salary Estimate'].apply(lambda x: 1 if 'employer provided salary:' in x.lower() else 0)

df = df[df['Salary Estimate'] != '-1']
salary = df['Salary Estimate'].apply(lambda x: x.split('(')[0])
minus_Kd = salary.apply(lambda x: x.replace('K','').replace('$',''))

min_hr = minus_Kd.apply(lambda x: x.lower().replace('per hour','').replace('employer provided salary:',''))

df['min_salary'] = min_hr.apply(lambda x: int(x.split('-')[0]))
df['max_salary'] = min_hr.apply(lambda x: int(x.split('-')[1]))
df['avg_salary'] = (df.min_salary+df.max_salary)/2

#Company name text only
df['company_txt'] = df.apply(lambda x: x['Company Name'] if x['Rating'] <0 else x['Company Name'][:-3], axis = 1)

#state field 
df['job_state'] = df['Location'].apply(lambda x: x.split(',')[1])
df.job_state.value_counts()

df['same_state'] = df.apply(lambda x: 1 if x.Location == x.Headquarters else 0, axis = 1)

#age of company 
df['age'] = df.Founded.apply(lambda x: x if x <1 else 2020 - x)

#parsing of job description (python, etc.)

#python
df['python_yn'] = df['Job Description'].apply(lambda x: 1 if 'python' in x.lower() else 0)
 
#r studio 
df['R_yn'] = df['Job Description'].apply(lambda x: 1 if 'r studio' in x.lower() or 'r-studio' in x.lower() else 0)
df.R_yn.value_counts()

#spark 
df['spark'] = df['Job Description'].apply(lambda x: 1 if 'spark' in x.lower() else 0)
df.spark.value_counts()

#aws 
df['aws'] = df['Job Description'].apply(lambda x: 1 if 'aws' in x.lower() else 0)
df.aws.value_counts()

#excel
df['excel'] = df['Job Description'].apply(lambda x: 1 if 'excel' in x.lower() else 0)
df.excel.value_counts()

df.columns

df_out = df.drop(['Unnamed: 0'], axis =1)

df_out.to_csv('salary_data_cleaned.csv',index = False)   

import pandas as pd 
import matplotlib.pyplot as plt 
import seaborn as sns 

df = pd.read_csv('salary_data_cleaned.csv')

df.head()
df.columns

def title_simplifier(title):
    if 'data scientist' in title.lower():
        return 'data scientist'
    elif 'data engineer' in title.lower():
        return 'data engineer'
    elif 'analyst' in title.lower():
        return 'analyst'
    elif 'machine learning' in title.lower():
        return 'mle'
    elif 'manager' in title.lower():
        return 'manager'
    elif 'director' in title.lower():
        return 'director'
    else:
        return 'na'
    
def seniority(title):
    if 'sr' in title.lower() or 'senior' in title.lower() or 'sr' in title.lower() or 'lead' in title.lower() or 'principal' in title.lower():
            return 'senior'
    elif 'jr' in title.lower() or 'jr.' in title.lower():
        return 'jr'
    else:
        return 'na'

df['job_simp'] = df['Job Title'].apply(title_simplifier)

df.job_simp.value_counts()

df['seniority'] = df['Job Title'].apply(seniority)
df.seniority.value_counts()

df['job_state']= df.job_state.apply(lambda x: x.strip() if x.strip().lower() != 'los angeles' else 'CA')
df.job_state.value_counts()

df['desc_len'] = df['Job Description'].apply(lambda x: len(x))
df['desc_len']

df['num_comp'] = df['Competitors'].apply(lambda x: len(x.split(',')) if x != '-1' else 0)
df['Competitors']
df['min_salary'] = df.apply(lambda x: x.min_salary*2 if x.hourly ==1 else x.min_salary, axis =1)
df['max_salary'] = df.apply(lambda x: x.max_salary*2 if x.hourly ==1 else x.max_salary, axis =1)
df[df.hourly ==1][['hourly','min_salary','max_salary']]
df['company_txt'] = df.company_txt.apply(lambda x: x.replace('\n', ''))
df['company_txt']
df.describe()
df.columns
df.Rating.hist()
df.avg_salary.hist()
df.age.hist()
df.desc_len.hist()
df.boxplot(column = ['age','avg_salary','Rating'])
df.boxplot(column = 'Rating')
df[['age','avg_salary','Rating','desc_len']].corr()
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(df[['age','avg_salary','Rating','desc_len','num_comp']].corr(),vmax=.3, center=0, cmap=cmap,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})

df.columns

df_cat = df[['Location', 'Headquarters', 'Size','Type of ownership', 'Industry', 'Sector', 'Revenue', 'company_txt', 'job_state','same_state', 'python_yn', 'R_yn',
       'spark', 'aws', 'excel', 'job_simp', 'seniority']]

for i in df_cat.columns:
    cat_num = df_cat[i].value_counts()
    print("graph for %s: total = %d" % (i, len(cat_num)))
    chart = sns.barplot(x=cat_num.index, y=cat_num)
    chart.set_xticklabels(chart.get_xticklabels(), rotation=90)
    plt.show()

for i in df_cat[['Location','Headquarters','company_txt']].columns:
    cat_num = df_cat[i].value_counts()[:20]
    print("graph for %s: total = %d" % (i, len(cat_num)))
    chart = sns.barplot(x=cat_num.index, y=cat_num)
    chart.set_xticklabels(chart.get_xticklabels(), rotation=90)
    plt.show()
    
df.columns

pd.pivot_table(df, index = 'job_simp', values = 'avg_salary')

pd.pivot_table(df, index = ['job_simp','seniority'], values = 'avg_salary')

pd.pivot_table(df, index = ['job_state','job_simp'], values = 'avg_salary').sort_values('job_state', ascending = False)

pd.options.display.max_rows
pd.set_option('display.max_rows', None)

pd.pivot_table(df, index = ['job_state','job_simp'], values = 'avg_salary', aggfunc = 'count').sort_values('job_state', ascending = False)

pd.pivot_table(df[df.job_simp == 'data scientist'], index = 'job_state', values = 'avg_salary').sort_values('avg_salary', ascending = False)

df.columns

df_pivots = df[['Rating', 'Industry', 'Sector', 'Revenue', 'num_comp', 'hourly', 'employer_provided', 'python_yn', 'R_yn', 'spark', 'aws', 'excel', 'Type of ownership','avg_salary']]

for i in df_pivots.columns:
    print(i)
    print(pd.pivot_table(df_pivots,index =i, values = 'avg_salary').sort_values('avg_salary', ascending = False))
    
pd.pivot_table(df_pivots, index = 'Revenue', columns = 'python_yn', values = 'avg_salary', aggfunc = 'count')

from wordcloud import WordCloud, ImageColorGenerator, STOPWORDS
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

words = " ".join(df['Job Description'])

def punctuation_stop(text):
    """remove punctuation and stop words"""
    filtered = []
    stop_words = set(stopwords.words('english'))
    word_tokens = word_tokenize(text)
    for w in word_tokens:
        if w not in stop_words and w.isalpha():
            filtered.append(w.lower())
    return filtered


words_filtered = punctuation_stop(words)

text = " ".join([ele for ele in words_filtered])

wc= WordCloud(background_color="white", random_state=1,stopwords=STOPWORDS, max_words = 2000, width =800, height = 1500)
wc.generate(text)

plt.figure(figsize=[10,10])
plt.imshow(interpolation="bilinear")
plt.axis('off')
plt.show()          

## Pipe Building
import pandas as pd 
import numpy as np

from sklearn.pipeline import Pipeline,FeatureUnion
from sklearn.base import BaseEstimator, TransformerMixin

class VarSelector(BaseEstimator, TransformerMixin):
    
    def __init__(self,var_names,drop_var=False):
        self.vars=var_names
        self.drop_var=drop_var
    
    def fit(self,x,y=None):
        return self
    
    def transform(self,X):
        if self.drop_var:
            return X.drop(self.vars,1)
        else:
            return X[self.vars]



class string_clean(BaseEstimator, TransformerMixin):
    
    def __init__(self,replace_it='',replace_with=''):
        self.replace_it=replace_it
        self.replace_with=replace_with
    
    def fit(self,x,y=None):
        return self
    
    def transform(self,X):
        for col in X.columns:
            X[col]=X[col].str.replace(self.replace_it,self.replace_with)
        return X



class convert_to_numeric(BaseEstimator, TransformerMixin):
    
    def __init__(self):
        pass
    
    def fit(self,x,y=None):
        return self
    
    def transform(self,X):
        for col in X.columns:
            X[col]=pd.to_numeric(X[col],errors='coerce')
        return X


class get_dummies_Pipe(BaseEstimator, TransformerMixin):
    
    def __init__(self,freq_cutoff=0):
        self.freq_cutoff=freq_cutoff
        self.var_cat_dict={}
        
    def fit(self,x,y=None):
        data_cols=x.columns

        for col in data_cols:
            k=x[col].value_counts()
            if (k<=self.freq_cutoff).sum()==0:
                cats=k.index[k>self.freq_cutoff][:-1]

            else:
                cats=k.index[k>self.freq_cutoff]

            self.var_cat_dict[col]=cats
        return self
            
    def transform(self,x,y=None):
        dummy_data=x.copy()
        for col in self.var_cat_dict.keys():
            for cat in self.var_cat_dict[col]:
                name=col+'_'+cat
                dummy_data[name]=(dummy_data[col]==cat).astype(int)
            del dummy_data[col]
        return dummy_data

        


class custom_fico(BaseEstimator, TransformerMixin):
    
    def __init__(self):
        pass
    
    def fit(self,x,y=None):
        return self
    
    def transform(self,X):
        k=X['FICO.Range'].str.split("-",expand=True).astype(float)
        X['fico']=0.5*(k[0]+k[1])
        del X['FICO.Range']
        return X



## Model Building 
import pandas as pd
file=r'/Users/Monica/Desktop/salary_prediction_app/eda_data.csv'
train=pd.read_csv(file)
train=train.iloc[:,1:]
import re
k=[re.sub('[$A-Za-z\(\)\.:]','',x) for x in train['Salary Estimate']]
train['Salary Estimate']=pd.Series(k).str.split('-',expand=True).astype(float).mean(axis=1)
train['Company Name']=train['Company Name'].str.split('\n',expand=True).iloc[:,0]
train.columns
drop_cols=['Job Description','seniority','job_simp','min_salary','max_salary','avg_salary','company_txt','desc_len']
train.drop(drop_cols,1,inplace=True)
train.shape
train['Size'].value_counts()
train.to_csv('simple_data.csv',index=False)
from mypipes import *
cat_vars=list(train.select_dtypes(['object']).columns)
num_vars=list(train.select_dtypes(exclude=['object']).columns)
num_vars.remove('Salary Estimate')num_vars=list(train.select_dtypes(exclude=['object']).columns)
num_vars.remove('Salary Estimate')
cat_vars
cat_pipe=Pipeline([
    ('cat_var_select',VarSelector(cat_vars)),
    ('create_dummies',get_dummies_Pipe(74))
])
data_pipe=FeatureUnion([
    ('num_vars',VarSelector(num_vars)),
    ('cat_data',cat_pipe)
])
from sklearn.linear_model import LinearRegression
complete_pipe=Pipeline([
    ('data_pipe',data_pipe),
    ('model',LinearRegression(fit_intercept=True))
])
target='Salary Estimate'
y=train[target]
x=train.drop(target,1)

complete_pipe.fit(x,y)

from sklearn.externals import joblib
joblib.dump(complete_pipe,'model_pipeline.pkl')

x[num_vars].describe()

x.columns                

import streamlit as st
import pandas as pd

st.title('Predicting Salaries')

default_dd=pd.read_csv('/Users/Monica/Desktop/salary_prediction_app/simple_data.csv')

_job_title=list(default_dd['Job Title'].unique())
_company_name=list(default_dd['Company Name'].unique())
_location=list(default_dd['Location'].unique())
_hq=list(default_dd['Headquarters'].unique())
_size=list(default_dd['Size'].unique())
_ownership=list(default_dd['Type of ownership'].unique())
_industry=list(default_dd['Industry'].unique())
_sector=list(default_dd['Sector'].unique())
_revenue=list(default_dd['Revenue'].unique())
_competitors=list(default_dd['Competitors'].unique())
_job_state=list(default_dd['job_state'].unique())

job_title=st.selectbox('Select Job Title',options=_job_title)
company_name=st.selectbox('Select Company Nmae',options=_company_name)
location=st.selectbox('Select Location',options=_location)
hq=st.selectbox('Select Headquarters',options=_hq)
size=st.selectbox('Select Size',options=_size)
ownership=st.selectbox('Select Type of ownership',options=_ownership)
industry=st.selectbox('Select Industry',options=_industry)
sector=st.selectbox('Select Sector',options=_sector)
revenue=st.selectbox('Select Revenue',options=_revenue)
competitors=st.selectbox('Select Competitors',options=_competitors)
job_state=st.selectbox('Select job_state',options=_job_state)

rating=st.slider('Select Rating',float(default_dd['Rating'].min()),float(default_dd['Rating'].max()))
founded=st.slider('Select Founded',float(default_dd['Founded'].min()),float(default_dd['Founded'].max()))
hourly=st.selectbox('Select hourly',options=[0,1])
employer_provided=st.selectbox('Select employer_provided',options=[0,1])
same_state=st.selectbox('Select same_state',options=[0,1])
age=founded=st.slider('Select age',float(default_dd['age'].min()),float(default_dd['age'].max()))
python_yn=st.selectbox('Select python_yn',options=[0,1])
R_yn=st.selectbox('Select R_yn',options=[0,1])
spark=st.selectbox('Select spark',options=[0,1])
aws=st.selectbox('Select aws',options=[0,1])
excel=st.selectbox('Select excel',options=[0,1])
num_comp=st.selectbox('Select num_comp',options=list(default_dd['num_comp'].unique()))


x=pd.DataFrame({'Job Title':[job_title], 
	'Rating':[rating], 
	'Company Name':[company_name], 
	'Location':[location], 
	'Headquarters':[hq],
    'Size':[size], 
    'Founded':[founded], 
    'Type of ownership':[ownership], 
    'Industry':[industry], 
    'Sector':[sector], 
    'Revenue':[revenue],
    'Competitors':[competitors],
    'hourly':[hourly], 
    'employer_provided':[employer_provided], 
    'job_state':[job_state], 
    'same_state':[same_state],
    'age':[age], 
    'python_yn':[python_yn], 
    'R_yn':[R_yn], 
    'spark':[spark], 
    'aws':[aws], 
    'excel':[excel], 
    'num_comp':[num_comp]})

from sklearn.externals import joblib

model=open('model_pipeline.pkl','rb')
model=joblib.load(model)

st.title('Your Predicted Salary is :')
st.info('$'+str(int(model.predict(x)[0]))+'K')